#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" Defines a file upload/download server for the given directory

Returns a JSON encoded list of files upon an empty GET request
  It is a dict of list of dicts.
    Keys are the filenames
    Each element of the list is a dict which contains the title,
    description, timestamp hash, and id of that file (in reverse
    chronological order)
Returns the latest file given a GET with a filename.
Returns a specific version given a GET with a filename and a query
string of version=timestamp

Write a file upon POST
Define various overwrite behavior with write_out callables
Returns the timestamp upon successful write; if for no other reason than it
makes decent doc tests possible. :)

WARNING: Changing the writeout behavior from "version" to "overwrite" will
delete all previous versions of a document, without asking for confirmation.
It should leave the disk and database in a consistent state, but this has
not been tested yet.

NOTE: The sprinkling of calls to conn.close() is to try to free up the sqlite3
file so it can be deleted on Windows during tests. Not there yet, but it does
seem to be helping somewhat.

Note: Timestamp form is: YYYYMMDDHHmmu
Where u is a six character, zero padded string representing the number
of microseconds.

    >>> from string import Template
    >>> from StringIO import StringIO
    >>> from tack.lib import default_env, run_app
    >>> from tack.lib.formreader import formreader
    >>> from tack.lib.jsontools import json_loads
    >>> import time
    >>> import os
    >>> from os.path import sep
    >>> from glob import glob

    >>> tmp_dir = filejoin(sep, 'tmp', 'fs')
    
    >>> if not os.path.exists(tmp_dir):
    ...     os.makedirs(tmp_dir)

First, we're testing the versioning writeout

    >>> app = formreader(filestorage(tmp_dir,writeout_version), keep_name=True)

    >>> env = default_env({'wsgi.input':file('tests/filestorage.test.data'),
    ...                    'REQUEST_METHOD':'POST',
    ...                    'CONTENT_LENGTH': '528',
    ...                    'CONTENT_TYPE':'multipart/form-data; boundary=---------------------------9304734438996164231261310786'})

    >>> ts1 = str(json_loads(run_app(app, env)[1][0]))
    >>> time.sleep(0.1)

    >>> second_test_data = Template(file('tests/filestorage.test.data').read()
    ...    ).substitute({'data':'Second Test'})

    >>> env = default_env({'wsgi.input':StringIO(second_test_data),
    ...                    'REQUEST_METHOD':'POST',
    ...                    'CONTENT_LENGTH': '532',
    ...                    'CONTENT_TYPE':'multipart/form-data; boundary=---------------------------9304734438996164231261310786'})

    >>> ts2 = str(json_loads(run_app(app, env)[1][0]))

    >>> env = default_env()
    >>> file_list = json_loads(run_app(app, env)[1][0])
    >>> u'thetest.file' in file_list
    True

    >>> str(file_list[u'thetest.file'][0][u'timestamp']) == ts2
    True

    >>> str(file_list[u'thetest.file'][1][u'timestamp']) == ts1
    True

    >>> env = default_env({'PATH_INFO':'/path/to/filestorage/thetest.file'})
    >>> run_app(app, env)[1][0]
    'A file. Second Test'

    >>> env = default_env({'PATH_INFO':'/path/to/filestorage/thetest.file',
    ...               'QUERY_STRING':'timestamp=%s' % ts1})
    >>> run_app(app, env)[1][0]
    'A file. ${data}'

    >>> for f in glob(filejoin(tmp_dir, '*')):
    ...                unlink(f)

Now we'll test the overwrite writeout

    >>> app = formreader(filestorage(tmp_dir, writeout_overwrite), keep_name=True)
    >>> data = file('tests/filestorage.test.data').read()

    >>> env = default_env({'wsgi.input':file('tests/filestorage.test.data'),
    ...                    'REQUEST_METHOD':'POST',
    ...                    'CONTENT_LENGTH': '528',
    ...                    'CONTENT_TYPE':'multipart/form-data; boundary=---------------------------9304734438996164231261310786'})

    >>> ts1 = str(json_loads(run_app(app, env)[1][0]))
    >>> time.sleep(0.1)

    >>> env = default_env({'wsgi.input':file('tests/filestorage.test.data'),
    ...                    'REQUEST_METHOD':'POST',
    ...                    'CONTENT_LENGTH': '528',
    ...                    'CONTENT_TYPE':'multipart/form-data; boundary=---------------------------9304734438996164231261310786'})

    >>> ts2 = str(json_loads(run_app(app, env)[1][0]))

Getting the first file should fail since it was overwritten

    >>> env = default_env({'PATH_INFO':'/path/to/filestorage/thetest.file',
    ...               'QUERY_STRING':'timestamp=%s' % ts1})
    >>> run_app(app, env)[0][0]
    '404 Not Found'
    
    >>> for f in glob(filejoin(tmp_dir, '*')):
    ...                unlink(f)    

Now to test the throw error writeout.

    >>> app = formreader(filestorage(tmp_dir,writeout_abort), keep_name=True)
    >>> data = file('tests/filestorage.test.data').read()

    >>> env = default_env({'wsgi.input':file('tests/filestorage.test.data'),
    ...                    'REQUEST_METHOD':'POST',
    ...                    'CONTENT_LENGTH': '528',
    ...                    'CONTENT_TYPE':'multipart/form-data; boundary=---------------------------9304734438996164231261310786'})

    >>> ts1 = str(json_loads(run_app(app, env)[1][0]))
    >>> time.sleep(0.1)

    >>> env = default_env({'wsgi.input':file('tests/filestorage.test.data'),
    ...                    'REQUEST_METHOD':'POST',
    ...                    'CONTENT_LENGTH': '528',
    ...                    'CONTENT_TYPE':'multipart/form-data; boundary=---------------------------9304734438996164231261310786'})

    >>> run_app(app, env)
    Traceback (most recent call last):
    ...
    OSError: File thetest.file exists

    >>> for f in glob(filejoin(tmp_dir,'*')):
    ...                unlink(f)
    >>> os.removedirs('/tmp/fs')

"""

import datetime
import sqlite3
from hashlib import sha512

from functools import partial
from os.path import (join as filejoin,
                     exists as fileexists,
                     split as pathsplit)
from os import unlink
from glob import glob
from paste.fileapp import FileApp
from tack.lib import content_headers, not_found, json_app

meta_table = 'meta_info'
datetime_format = '%Y%m%d%H%M%S'

def writeout(dest_dir, form, db_conn, on_exist):
    cur = db_conn.cursor()
    filename, file_obj = form['fileupload']
    title = form['title']
    description = form['description']
    hash = sha512(file_obj.read()).hexdigest()
    file_obj.seek(0)

    timestamp = check_exists(db_conn, dest_dir, filename)

    if timestamp:
        if on_exist == 'overwrite':
            cur.execute("DELETE from %s where filename = ?" % meta_table,
                    (filename,))
            try:
                for f in glob(filejoin(dest_dir, filename + '*')):
                    unlink(f)
            except:
                con.rollback()
                raise

        elif on_exist == 'abort':
            raise OSError('File %s exists' % filename)

        elif on_exist == 'version':
            # We don't need to do anything, the database already 
            # versions by default.
            pass
        else:
            raise NotImplementedError('Cannot handle file by %s' % on_exist)

    try:
        t = datetime.datetime.now()
        dt_string = (t.strftime(datetime_format)
                     + str(t.microsecond).rjust(6, '0'))

        cur.execute("INSERT INTO %s (filename, title, description, "
                    "timestamp, hash) VALUES(?,?,?,?,?)"
        % meta_table, (filename, title, description, dt_string, hash))

        file(filejoin(dest_dir,filename + '.' + dt_string),
             'wb').write(file_obj.read())
    except (IOError, OSError), e:
        con.rollback()
        raise Exception("Could not write file: %s", str(e))
    except:
        db_conn.rollback()
        raise
    else:
        db_conn.commit()
    finally:
        db_conn.close()

    return dt_string

writeout_overwrite = partial(writeout, on_exist='overwrite')
writeout_abort = partial(writeout, on_exist='abort')
writeout_version = partial(writeout, on_exist='version')

def connect(upload_directory):
    conn = sqlite3.connect(filejoin(upload_directory, 'file_meta.sqlite3'))

    cur = conn.cursor()

    exists = cur.execute("select count(*) from sqlite_master where name = '%s'"
                         % meta_table).fetchone()[0]

    if not exists:
        cur.execute("CREATE TABLE %s ("
        "id INTEGER PRIMARY KEY AUTOINCREMENT,"
        "filename TEXT,"
        "title TEXT,"
        "description TEXT,"
        "timestamp TEXT,"
        "hash TEXT"
        ")" % meta_table)

        cur.execute("CREATE UNIQUE INDEX version_idx "
                    "ON %s (filename, timestamp)" % meta_table)

        conn.commit()

    return conn

def check_exists(conn, upload_directory, filename, timestamp = None):
    cur = conn.cursor()

    if timestamp:
        cur.execute("SELECT timestamp FROM %s WHERE filename = ? "
                    "AND timestamp = ? ORDER BY id DESC" %
                    meta_table, (filename, timestamp))
    else:
        cur.execute("SELECT timestamp FROM %s WHERE filename = ? "
                    "ORDER BY id DESC LIMIT 1" %
                    meta_table, (filename,))
    try:
        timestamp = cur.fetchone()[0]
        if not fileexists(filejoin(upload_directory, filename + '.' + timestamp)):
            raise AssertionError('Database inconsistency.  Database says %s with '
                                 'timestamp %s exists, but it does not exist on disk.'
                                 % (filename, timestamp))

    except (TypeError, IndexError):
        timestamp = None

    return timestamp

def get_file_info(conn, filename, timestamp=None):
    cur = conn.cursor()
    file_info = []

    if timestamp:
        cur.execute("""SELECT id, filename, title, description, timestamp,
        hash FROM %s WHERE filename = ? AND timestamp = ? ORDER BY id DESC"""
                   % meta_table, (filename, timestamp))
    else:
        cur.execute("""SELECT id, filename, title, description, timestamp,
        hash FROM %s WHERE filename = ? ORDER BY id DESC LIMIT 1"""
                   % (meta_table), (filename, ))

    for id, filename, title, description, timestamp, hash in cur:
        file_info.append({'title':title,
                          'description':description,
                          'timestamp':timestamp,
                          'id':id,
                          'hash':hash})

    return file_info

def get_dir_info(conn):
    cur = conn.cursor()
    dir_info = {}

    cur.execute("""SELECT id, filename, title, description, timestamp, hash
    FROM %s ORDER BY filename, id DESC""" % meta_table)

    for id, filename, title, description, timestamp, hash in cur:
        if filename not in dir_info:
            dir_info[filename] = []

        dir_info[filename].append({'title':title,
                                   'description':description,
                                   'timestamp':timestamp,
                                   'hash':hash,
                                   'id':id})
    conn.close()
    return dir_info

def filestorage(upload_directory, writeout_method=writeout_overwrite,
                handle404=not_found, fileapp=FileApp):
    def filestorage_app(environ, start_response):

        db_conn = connect(upload_directory)

        if environ['REQUEST_METHOD'] == 'POST':
            return json_app(
                    writeout_method(
                        upload_directory,
                        environ['form'],
                        db_conn))(environ, start_response)
        else:
            path_info, filename = pathsplit(environ['PATH_INFO'])
            if filename:

                if 'timestamp' in environ['form']:
                    timestamp = environ['form']['timestamp']
                    real_filename = filename + '.' + timestamp
                else:
                    timestamp = None
                    real_filename = filename

                ts = check_exists(db_conn, upload_directory, filename, timestamp)

                if ts:
                    if not timestamp:
                        real_filename = filename + '.' + ts

                    file_meta = get_file_info(db_conn, filename, timestamp)

                    if (file_meta[0]['hash'] !=
                        sha512(file(
                                    filejoin(upload_directory, real_filename),
                                    'rb').read()).hexdigest()):
                        raise AssertionError('Filename %s does not agree with '
                                             'the stored hash' % real_filename)

                    db_conn.close()
                    return fileapp(filejoin(upload_directory,real_filename),
                               content_disposition='inline; filename=%s' % filename)(environ, start_response)
                else:
                    db_conn.close()
                    return handle404(environ, start_response)
            else:
                return json_app(get_dir_info(db_conn))(environ, start_response)

    return filestorage_app
